# -*- coding: utf-8 -*-
"""
-------------------------------------------------
   File Name：     qianniu_run
   Description : 千牛采集
   Author :       zm.z
   date：          2020/2/3
-------------------------------------------------
   Change Activity:
                   2020/2/3:
-------------------------------------------------
"""
__author__ = 'zm.z'
import re
import time
import json
import jsonpath
import urllib.parse
import random
import hashlib
import requests
import os
print(os.path.abspath(os.curdir))
import sys
sys.path.append(os.path.abspath(os.curdir))
from tmall.proporties import qianniu_const as Const
from tmall.proporties.db import session
from tmall.proporties.models import QianniuDouyinFlow
from tmall.proporties.rdb import ToolRedis
from tmall.proporties.decorate import saveFuncExit
session = session()
taskName = "qianniu:task"

def get_cookies(i):
    cookies = "t=cf2c0befa1e78ced7d387629d8b8e6b4; _m_h5_tk=6066e40d205e824ab85508a2e47ba656_1580809267815; _m_h5_tk_enc=1e3e5407e806680699ec08ac2d0d11b7; cookie2=5b76c541373424f23456b6453549479a; _tb_token_=77f5f83947bb3; uc1=cookie14=UoTUOqiueDcngg%3D%3D&lng=zh_CN; _samesite_flag_=true; x=3375170974; csg=ab6c1046; skt=65697eb6c35b9966; unb=2200778572057; sn=perfectdiary%E6%97%97%E8%88%B0%E5%BA%97%3Aspider; isg=BMvLC4UFEatQ7E1wz2k8hSG3WW-1YN_i2loblT3JMYsjXO--xTNJMh--Ntyy5zfa; l=cBr4H-8PQMACeJfLBOfwCuI8as79uIdVGkPzw4_GjICPOeX9qccFWZ0sDQtpCn1VK6eyR38eHdDJBRTGHyC26e4Y8k8ewa5..; cna=kdC9Fs3ZsTECATspotwtpHk7; v=0; x5sec=7b22617365727665723b32223a223934663533333464343337663862306666373830306366613938383163646231434c2b7935504546454c4c70304e2b6c792b666376514561447a49794d4441334e7a67314e7a49774e5463374d513d3d227d"
    # cookies = rdb.rdb.hget("ali_cookies", "qianniu")
    # if isinstance(cookies, bytes):
    #     cookies = cookies.decode()
    return cookies

def pack_params(prefix, api, v, data):
    url = "%(prefix)s/%(api)s/%(v)s/" % {"prefix": prefix,
                                         "api": api,
                                         "v": v}
    headers = Const.cHEADERS
    headers['Cookie'] = get_cookies(cid)
    token = re.search("_m_h5_tk=([a-zA-Z0-9]{32})", headers['Cookie'])
    assert token, "缺少token参数，登录异常"
    token = token.group(1)

    t = str(int(time.time() * 1000))
    r = "%(token)s&%(t)s&%(appkey)s&%(data)s" % {"token":token, "t":t, "appkey":Const.cAPPKEY,
                                                 "data":data}
    md = hashlib.md5()
    md.update(r.encode())
    r = md.hexdigest()

    params = {
        "jsv": Const.cJSV,
        "appKey": Const.cAPPKEY,
        "t": str(t),
        "v": v,
        "type":"jsonp",
        "dataType":"jsonp",
        "callback":"mtopjsonp%d" % random.randint(15,50),
        "api":api,
        "sign":r,
        "data":data
    }
    url = url + "?" + urllib.parse.urlencode(params)
    return {
        "url":url,
        "header":headers,
    }



def Fetch(request_data):
    count = 0
    while count < 3:
        try:
            response = requests.get(
            url=request_data['url'],
            headers=request_data['header'],
            timeout=10
            )
            break
        except requests.exceptions.ConnectionError or requests.exceptions.ConnectTimeout :
            count += 1
            time.sleep(10)
    else:
        raise ValueError("happen error when request %s" % request_data['url'])


    if "login" in response.url:
        raise ValueError("登录失效, 需重新登录")
    content = re.sub("mtopjsonp\d\d.|.$", "", response.text)
    jsn_content = json.loads(content)
    print(jsn_content)
    assert "SUCCESS::调用成功" in jsn_content['ret'] or 'SUCCESS::获取推广数据成功' in jsn_content['ret'], \
        jsn_content['ret'][0]
    return jsn_content


@saveFuncExit
def FetchCateLst(page=1):
    qn_cate = taskName+":cateLst"
    qn_itm = taskName+":itmLst"
    while True:
        page = rdb.pop(taskName+":cateLst")
        if page is None:
            page = 1
        else:
            page = page['page']

        try:
            data = "{\"page\":%d,\"pageSize\":15}" % page
            request_data = pack_params(Const.cPrefixURL, Const.cCateAPI, Const.cV,  data)
            jsn_content = Fetch(request_data)

            data = jsonpath.jsonpath(jsn_content, "$..list[*]") or []
            for i in data:
                print(i)
                if i['activityName'].startswith("快手信息流") or i['activityName'].startswith("快手信息流") or i['activityName'].startswith("KS") :
                    print("[-]", i)
                    # FetchItmLst(i['id'])
                    rdb.push(qn_itm, {"activity_id":i['id'], "activity_name":i['activityName']})

            pages= jsonpath.jsonpath(jsn_content, "$..pages")
            if isinstance(pages, bool):
                return

            if page <= int(pages[0]):
                # time.sleep(5)
                # FetchCateLst(page=page+1)
                rdb.push(qn_cate, {"page":page+1})
            else:
                return
        except Exception as e:
            rdb.push(qn_cate, {"page": page })
            raise ValueError(e)


@saveFuncExit
def FetchItmLst():
    qn_itm = taskName+":itmLst"
    qn_data = taskName+":dataLst"
    while True:
        try:
            activity = rdb.pop(qn_itm)
            if activity is None:
                return

            activity_id=activity['activity_id']
            data = "{\"activityId\":%s}" % activity_id
            request_data = pack_params(Const.cPrefixURL, Const.cItmAPI, Const.cV, data)
            jsn_content = Fetch(request_data)

            data = jsonpath.jsonpath(jsn_content, "$..module[*]") or []
            for i in data:
                context = json.loads(i['context'])
                title = context.get('cellTitle') or context.get('title')
                kwargs = {
                    "activity_id":i['activityId'],
                    "ext_id":i['id'],
                    "cell_title":title,
                    'gmt_create':i['gmtCreate'],
                    'gmt_modified':i['gmtModified'],
                    'item_id':i['itemId'],
                    'main_accountId':i['mainAccountId'],
                    'status':i['status'],
                    'tkl_url':i['tklUrl'],
                    'activity_name':activity['activity_name']
                }
                rdb.push(qn_data, dict(spmb=context['spmb'], sellerid=i['mainAccountId'], kwargs=kwargs))
        except Exception as e:
            rdb.push(qn_itm, activity)
            raise ValueError(e)



@saveFuncExit
def FetchData():
    qn_data = taskName+":dataLst"
    count = 0
    datas = []
    while True:
        try:
            req_data = rdb.pop(qn_data)
            if req_data is None:
                if datas:
                    session.execute(QianniuDouyinFlow.__table__.insert(), datas)
                    session.commit()
                break
            sellerid, spmb, kwargs = req_data['sellerid'], req_data['spmb'], req_data['kwargs']
            data = "{\"seller_id\":%s,\"spmb\":\"%s\"}" % (sellerid, spmb)
            request_data = pack_params(Const.cPrefixURL, Const.cDataAPI, Const.cV, data)
            jsn_content = Fetch(request_data)
            data = jsonpath.jsonpath(jsn_content, "$..module")[0]
            data.update(kwargs)
            datas.append(data)
            count += 1

            if count > 30:
                count = 0
                session.execute(QianniuDouyinFlow.__table__.insert(), datas)
                session.commit()
                datas = []

            print("[-]", data)
        except Exception as e:
            session.execute(QianniuDouyinFlow.__table__.insert(), datas)
            session.commit()
            rdb.push(qn_data, req_data)
            raise ValueError(e)





# FetchItmLst(9978481)
# FetchData("5-3375170974-GgBl8VEd4", "3375170974")
@saveFuncExit
def QianNiuRun():
    # 抓取活动列表
    FetchCateLst()

    # 根据每个活动采集里面的商品信息
    FetchItmLst()

    FetchData()

if __name__ == '__main__':
    from tmall.proporties.argsmodule import get_args
    # 获取命令行中的参数(账号, 密码, id)
    addargs_dict = {

        '-id': {'help': 'cookies id', 'type': str, 'default': None},
    }

    args = get_args(addargs_dict)
    # 登录

    cid = args.id

    if cid is None:
        print("`cid` must be not null")
    # else:
        try:
            rdb = ToolRedis()
            QianNiuRun()
        except Exception as e:
            print(e)